{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "id": "J9gt14qHb7LY",
    "outputId": "7b7c9774-f205-4365-b7d0-fd991d020ead"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "B1d2rddEJv_v"
   },
   "outputs": [],
   "source": [
    "# !rm -rf tf2.0-BERT/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 119
    },
    "colab_type": "code",
    "id": "Z32azGO3kVM8",
    "outputId": "9bf3b4fb-c1b2-43c0-9292-41f145499e73"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'tf2.0-BERT'...\n",
      "remote: Enumerating objects: 64, done.\u001b[K\n",
      "remote: Counting objects:   1% (1/64)\u001b[K\r",
      "remote: Counting objects:   3% (2/64)\u001b[K\r",
      "remote: Counting objects:   4% (3/64)\u001b[K\r",
      "remote: Counting objects:   6% (4/64)\u001b[K\r",
      "remote: Counting objects:   7% (5/64)\u001b[K\r",
      "remote: Counting objects:   9% (6/64)\u001b[K\r",
      "remote: Counting objects:  10% (7/64)\u001b[K\r",
      "remote: Counting objects:  12% (8/64)\u001b[K\r",
      "remote: Counting objects:  14% (9/64)\u001b[K\r",
      "remote: Counting objects:  15% (10/64)\u001b[K\r",
      "remote: Counting objects:  17% (11/64)\u001b[K\r",
      "remote: Counting objects:  18% (12/64)\u001b[K\r",
      "remote: Counting objects:  20% (13/64)\u001b[K\r",
      "remote: Counting objects:  21% (14/64)\u001b[K\r",
      "remote: Counting objects:  23% (15/64)\u001b[K\r",
      "remote: Counting objects:  25% (16/64)\u001b[K\r",
      "remote: Counting objects:  26% (17/64)\u001b[K\r",
      "remote: Counting objects:  28% (18/64)\u001b[K\r",
      "remote: Counting objects:  29% (19/64)\u001b[K\r",
      "remote: Counting objects:  31% (20/64)\u001b[K\r",
      "remote: Counting objects:  32% (21/64)\u001b[K\r",
      "remote: Counting objects:  34% (22/64)\u001b[K\r",
      "remote: Counting objects:  35% (23/64)\u001b[K\r",
      "remote: Counting objects:  37% (24/64)\u001b[K\r",
      "remote: Counting objects:  39% (25/64)\u001b[K\r",
      "remote: Counting objects:  40% (26/64)\u001b[K\r",
      "remote: Counting objects:  42% (27/64)\u001b[K\r",
      "remote: Counting objects:  43% (28/64)\u001b[K\r",
      "remote: Counting objects:  45% (29/64)\u001b[K\r",
      "remote: Counting objects:  46% (30/64)\u001b[K\r",
      "remote: Counting objects:  48% (31/64)\u001b[K\r",
      "remote: Counting objects:  50% (32/64)\u001b[K\r",
      "remote: Counting objects:  51% (33/64)\u001b[K\r",
      "remote: Counting objects:  53% (34/64)\u001b[K\r",
      "remote: Counting objects:  54% (35/64)\u001b[K\r",
      "remote: Counting objects:  56% (36/64)\u001b[K\r",
      "remote: Counting objects:  57% (37/64)\u001b[K\r",
      "remote: Counting objects:  59% (38/64)\u001b[K\r",
      "remote: Counting objects:  60% (39/64)\u001b[K\r",
      "remote: Counting objects:  62% (40/64)\u001b[K\r",
      "remote: Counting objects:  64% (41/64)\u001b[K\r",
      "remote: Counting objects:  65% (42/64)\u001b[K\r",
      "remote: Counting objects:  67% (43/64)\u001b[K\r",
      "remote: Counting objects:  68% (44/64)\u001b[K\r",
      "remote: Counting objects:  70% (45/64)\u001b[K\r",
      "remote: Counting objects:  71% (46/64)\u001b[K\r",
      "remote: Counting objects:  73% (47/64)\u001b[K\r",
      "remote: Counting objects:  75% (48/64)\u001b[K\r",
      "remote: Counting objects:  76% (49/64)\u001b[K\r",
      "remote: Counting objects:  78% (50/64)\u001b[K\r",
      "remote: Counting objects:  79% (51/64)\u001b[K\r",
      "remote: Counting objects:  81% (52/64)\u001b[K\r",
      "remote: Counting objects:  82% (53/64)\u001b[K\r",
      "remote: Counting objects:  84% (54/64)\u001b[K\r",
      "remote: Counting objects:  85% (55/64)\u001b[K\r",
      "remote: Counting objects:  87% (56/64)\u001b[K\r",
      "remote: Counting objects:  89% (57/64)\u001b[K\r",
      "remote: Counting objects:  90% (58/64)\u001b[K\r",
      "remote: Counting objects:  92% (59/64)\u001b[K\r",
      "remote: Counting objects:  93% (60/64)\u001b[K\r",
      "remote: Counting objects:  95% (61/64)\u001b[K\r",
      "remote: Counting objects:  96% (62/64)\u001b[K\r",
      "remote: Counting objects:  98% (63/64)\u001b[K\r",
      "remote: Counting objects: 100% (64/64)\u001b[K\r",
      "remote: Counting objects: 100% (64/64), done.\u001b[K\n",
      "remote: Compressing objects: 100% (44/44), done.\u001b[K\n",
      "remote: Total 64 (delta 13), reused 64 (delta 13), pack-reused 0\u001b[K\n",
      "Unpacking objects: 100% (64/64), done.\n"
     ]
    }
   ],
   "source": [
    "# Download the bert code from github.\n",
    "!git clone https://github.com/tensorflow/models.git\n",
    "# Download necessary files from my github.\n",
    "!git clone https://github.com/gnpdelta/tf2.0-BERT.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "vX4h5wYuke4k",
    "outputId": "ab1c7786-0028-48b0-bdaf-8b3936f946ff"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tf-nightly-gpu\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/76/46/cde0e2a7615c91841d60f4ee7e649f411c72d9a6adf032013d57b4be8dda/tf_nightly_gpu-2.2.0.dev20200407-cp36-cp36m-manylinux2010_x86_64.whl (517.2MB)\n",
      "\u001b[K     |████████████████████████████████| 517.2MB 29kB/s \n",
      "\u001b[?25hRequirement already satisfied: numpy<2.0,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (1.18.2)\n",
      "Requirement already satisfied: keras-preprocessing>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (1.1.0)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (3.2.0)\n",
      "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (0.34.2)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (1.1.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.8 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (0.2.0)\n",
      "Requirement already satisfied: h5py<2.11.0,>=2.10.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (2.10.0)\n",
      "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (3.10.0)\n",
      "Collecting tb-nightly<2.4.0a0,>=2.3.0a0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/63/a0b69e4c814581e52bfa76ec456fe7b910242b9497d8d57732ac1729da53/tb_nightly-2.3.0a20200406-py3-none-any.whl (2.9MB)\n",
      "\u001b[K     |████████████████████████████████| 2.9MB 31.0MB/s \n",
      "\u001b[?25hRequirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (1.12.0)\n",
      "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (0.9.0)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (1.27.2)\n",
      "Requirement already satisfied: scipy==1.4.1; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (1.4.1)\n",
      "Collecting tf-estimator-nightly\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e2/ab/9f69dda13e7bd6ac9afc988b922ff53f6cc29716974898a428d3e0531a84/tf_estimator_nightly-2.3.0.dev2020040801-py2.py3-none-any.whl (455kB)\n",
      "\u001b[K     |████████████████████████████████| 460kB 42.9MB/s \n",
      "\u001b[?25hRequirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (0.3.3)\n",
      "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (1.12.1)\n",
      "Requirement already satisfied: astunparse==1.6.3 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (1.6.3)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.9.2->tf-nightly-gpu) (46.1.3)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly-gpu) (1.6.0.post2)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly-gpu) (2.21.0)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly-gpu) (1.0.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly-gpu) (0.4.1)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly-gpu) (3.2.1)\n",
      "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly-gpu) (1.7.2)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly-gpu) (2.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly-gpu) (2019.11.28)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly-gpu) (3.0.4)\n",
      "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly-gpu) (1.24.3)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly-gpu) (1.3.0)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly-gpu) (0.2.8)\n",
      "Requirement already satisfied: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly-gpu) (4.0)\n",
      "Requirement already satisfied: cachetools<3.2,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly-gpu) (3.1.1)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly-gpu) (3.1.0)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.6/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tb-nightly<2.4.0a0,>=2.3.0a0->tf-nightly-gpu) (0.4.8)\n",
      "Installing collected packages: tb-nightly, tf-estimator-nightly, tf-nightly-gpu\n",
      "Successfully installed tb-nightly-2.3.0a20200406 tf-estimator-nightly-2.3.0.dev2020040801 tf-nightly-gpu-2.2.0.dev20200407\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 1)) (1.12.0)\n",
      "Requirement already satisfied: google-api-python-client>=1.6.7 in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 2)) (1.7.12)\n",
      "Requirement already satisfied: google-cloud-bigquery>=0.31.0 in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 3)) (1.21.0)\n",
      "Requirement already satisfied: kaggle>=1.3.9 in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 4)) (1.5.6)\n",
      "Collecting mlperf_compliance==0.0.10\n",
      "  Downloading https://files.pythonhosted.org/packages/f4/08/f2febd8cbd5c9371f7dab311e90400d83238447ba7609b3bf0145b4cb2a2/mlperf_compliance-0.0.10-py3-none-any.whl\n",
      "Requirement already satisfied: numpy>=1.15.4 in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 6)) (1.18.2)\n",
      "Requirement already satisfied: oauth2client>=4.1.2 in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 7)) (4.1.3)\n",
      "Requirement already satisfied: pandas>=0.22.0 in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 8)) (1.0.3)\n",
      "Requirement already satisfied: psutil>=5.4.3 in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 9)) (5.4.8)\n",
      "Collecting py-cpuinfo>=3.3.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/42/60/63f28a5401da733043abe7053e7d9591491b4784c4f87c339bf51215aa0a/py-cpuinfo-5.0.0.tar.gz (82kB)\n",
      "\u001b[K     |████████████████████████████████| 92kB 5.2MB/s \n",
      "\u001b[?25hRequirement already satisfied: scipy>=0.19.1 in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 11)) (1.4.1)\n",
      "Requirement already satisfied: tensorflow-hub>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 12)) (0.8.0)\n",
      "Collecting tensorflow-model-optimization>=0.2.1\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/09/7e/e94aa029999ec30951e8129fa992fecbbaffda66eba97c65d5a83f8ea96d/tensorflow_model_optimization-0.3.0-py2.py3-none-any.whl (165kB)\n",
      "\u001b[K     |████████████████████████████████| 174kB 11.1MB/s \n",
      "\u001b[?25hRequirement already satisfied: tensorflow-datasets in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 14)) (2.1.0)\n",
      "Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 15)) (0.8.3)\n",
      "Requirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 16)) (0.7)\n",
      "Requirement already satisfied: gin-config in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 17)) (0.3.0)\n",
      "Requirement already satisfied: typing in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 18)) (3.6.6)\n",
      "Collecting sentencepiece\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
      "\u001b[K     |████████████████████████████████| 1.0MB 40.7MB/s \n",
      "\u001b[?25hRequirement already satisfied: Cython in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 20)) (0.29.16)\n",
      "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 21)) (3.2.1)\n",
      "Collecting opencv-python-headless\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/43/2c/909a04b07360516953beaf6f66480bb6b84b817c6b300c1235bfb2901ad8/opencv_python_headless-4.2.0.34-cp36-cp36m-manylinux1_x86_64.whl (21.6MB)\n",
      "\u001b[K     |████████████████████████████████| 21.6MB 26.4MB/s \n",
      "\u001b[?25hRequirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 23)) (3.13)\n",
      "Requirement already satisfied: Pillow in /usr/local/lib/python3.6/dist-packages (from -r ./models/official/requirements.txt (line 24)) (7.0.0)\n",
      "Obtaining pycocotools from git+https://github.com/cocodataset/cocoapi#egg=pycocotools&subdirectory=PythonAPI (from -r ./models/official/requirements.txt (line 25))\n",
      "  Cloning https://github.com/cocodataset/cocoapi to ./src/pycocotools\n",
      "  Running command git clone -q https://github.com/cocodataset/cocoapi /content/src/pycocotools\n",
      "Requirement already satisfied: google-auth-httplib2>=0.0.3 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.6.7->-r ./models/official/requirements.txt (line 2)) (0.0.3)\n",
      "Requirement already satisfied: httplib2<1dev,>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.6.7->-r ./models/official/requirements.txt (line 2)) (0.17.1)\n",
      "Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.6.7->-r ./models/official/requirements.txt (line 2)) (3.0.1)\n",
      "Requirement already satisfied: google-auth>=1.4.1 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.6.7->-r ./models/official/requirements.txt (line 2)) (1.7.2)\n",
      "Requirement already satisfied: google-cloud-core<2.0dev,>=1.0.3 in /usr/local/lib/python3.6/dist-packages (from google-cloud-bigquery>=0.31.0->-r ./models/official/requirements.txt (line 3)) (1.0.3)\n",
      "Requirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.6/dist-packages (from google-cloud-bigquery>=0.31.0->-r ./models/official/requirements.txt (line 3)) (3.10.0)\n",
      "Requirement already satisfied: google-resumable-media!=0.4.0,<0.5.0dev,>=0.3.1 in /usr/local/lib/python3.6/dist-packages (from google-cloud-bigquery>=0.31.0->-r ./models/official/requirements.txt (line 3)) (0.4.1)\n",
      "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.6/dist-packages (from kaggle>=1.3.9->-r ./models/official/requirements.txt (line 4)) (2.8.1)\n",
      "Requirement already satisfied: python-slugify in /usr/local/lib/python3.6/dist-packages (from kaggle>=1.3.9->-r ./models/official/requirements.txt (line 4)) (4.0.0)\n",
      "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from kaggle>=1.3.9->-r ./models/official/requirements.txt (line 4)) (1.24.3)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from kaggle>=1.3.9->-r ./models/official/requirements.txt (line 4)) (2.21.0)\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from kaggle>=1.3.9->-r ./models/official/requirements.txt (line 4)) (4.38.0)\n",
      "Requirement already satisfied: certifi in /usr/local/lib/python3.6/dist-packages (from kaggle>=1.3.9->-r ./models/official/requirements.txt (line 4)) (2019.11.28)\n",
      "Requirement already satisfied: rsa>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.1.2->-r ./models/official/requirements.txt (line 7)) (4.0)\n",
      "Requirement already satisfied: pyasn1>=0.1.7 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.1.2->-r ./models/official/requirements.txt (line 7)) (0.4.8)\n",
      "Requirement already satisfied: pyasn1-modules>=0.0.5 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.1.2->-r ./models/official/requirements.txt (line 7)) (0.2.8)\n",
      "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.22.0->-r ./models/official/requirements.txt (line 8)) (2018.9)\n",
      "Collecting dm-tree~=0.1.1\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ee/47/948602fe82595056eb7f14b5005ee525c62a73218ffffe2aabb6b9e3ed42/dm_tree-0.1.4-cp36-cp36m-manylinux1_x86_64.whl (293kB)\n",
      "\u001b[K     |████████████████████████████████| 296kB 40.1MB/s \n",
      "\u001b[?25hRequirement already satisfied: attrs>=18.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->-r ./models/official/requirements.txt (line 14)) (19.3.0)\n",
      "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->-r ./models/official/requirements.txt (line 14)) (0.16.0)\n",
      "Requirement already satisfied: absl-py in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->-r ./models/official/requirements.txt (line 14)) (0.9.0)\n",
      "Requirement already satisfied: termcolor in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->-r ./models/official/requirements.txt (line 14)) (1.1.0)\n",
      "Requirement already satisfied: promise in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->-r ./models/official/requirements.txt (line 14)) (2.3)\n",
      "Requirement already satisfied: wrapt in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->-r ./models/official/requirements.txt (line 14)) (1.12.1)\n",
      "Requirement already satisfied: tensorflow-metadata in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->-r ./models/official/requirements.txt (line 14)) (0.21.1)\n",
      "Requirement already satisfied: dill in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->-r ./models/official/requirements.txt (line 14)) (0.3.1.1)\n",
      "Requirement already satisfied: typeguard in /usr/local/lib/python3.6/dist-packages (from tensorflow-addons->-r ./models/official/requirements.txt (line 15)) (2.7.1)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->-r ./models/official/requirements.txt (line 21)) (2.4.6)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->-r ./models/official/requirements.txt (line 21)) (0.10.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->-r ./models/official/requirements.txt (line 21)) (1.2.0)\n",
      "Requirement already satisfied: setuptools>=18.0 in /usr/local/lib/python3.6/dist-packages (from pycocotools->-r ./models/official/requirements.txt (line 25)) (46.1.3)\n",
      "Requirement already satisfied: cachetools<3.2,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.4.1->google-api-python-client>=1.6.7->-r ./models/official/requirements.txt (line 2)) (3.1.1)\n",
      "Requirement already satisfied: google-api-core<2.0.0dev,>=1.14.0 in /usr/local/lib/python3.6/dist-packages (from google-cloud-core<2.0dev,>=1.0.3->google-cloud-bigquery>=0.31.0->-r ./models/official/requirements.txt (line 3)) (1.16.0)\n",
      "Requirement already satisfied: text-unidecode>=1.3 in /usr/local/lib/python3.6/dist-packages (from python-slugify->kaggle>=1.3.9->-r ./models/official/requirements.txt (line 4)) (1.3)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->kaggle>=1.3.9->-r ./models/official/requirements.txt (line 4)) (3.0.4)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->kaggle>=1.3.9->-r ./models/official/requirements.txt (line 4)) (2.8)\n",
      "Requirement already satisfied: googleapis-common-protos in /usr/local/lib/python3.6/dist-packages (from tensorflow-metadata->tensorflow-datasets->-r ./models/official/requirements.txt (line 14)) (1.51.0)\n",
      "Building wheels for collected packages: py-cpuinfo\n",
      "  Building wheel for py-cpuinfo (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for py-cpuinfo: filename=py_cpuinfo-5.0.0-cp36-none-any.whl size=18684 sha256=b8a18236a16a62047d10539b1ef0a6e950f4810af15ec2ed76c6d219ce619c5f\n",
      "  Stored in directory: /root/.cache/pip/wheels/01/7e/a9/b982d0fea22b7e4ae5619de949570cde5ad55420cec16e86a5\n",
      "Successfully built py-cpuinfo\n",
      "Installing collected packages: mlperf-compliance, py-cpuinfo, dm-tree, tensorflow-model-optimization, sentencepiece, opencv-python-headless, pycocotools\n",
      "\u001b[33m  WARNING: The script cpuinfo is installed in '/root/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
      "  Found existing installation: pycocotools 2.0.0\n",
      "    Uninstalling pycocotools-2.0.0:\n",
      "      Successfully uninstalled pycocotools-2.0.0\n",
      "  Running setup.py develop for pycocotools\n",
      "Successfully installed dm-tree-0.1.4 mlperf-compliance-0.0.10 opencv-python-headless-4.2.0.34 py-cpuinfo-5.0.0 pycocotools sentencepiece-0.1.85 tensorflow-model-optimization-0.3.0\n"
     ]
    }
   ],
   "source": [
    "# Install tf-nightly and bert depencency\n",
    "# NOTE: Click \"RESTART RUNTIME\" button to restart the runtime environment after installing above.\n",
    "!pip install tf-nightly-gpu\n",
    "!pip install --user -r ./models/official/requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "lNFoDYVgoX4B",
    "outputId": "ed6c59e2-6960-42d9-f159-b4c33fc0bff4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Uninstalling tensorboard-2.2.0:\n",
      "  Would remove:\n",
      "    /usr/local/bin/tensorboard\n",
      "    /usr/local/lib/python3.6/dist-packages/tensorboard-2.2.0.dist-info/*\n",
      "    /usr/local/lib/python3.6/dist-packages/tensorboard/*\n",
      "  Would not remove (might be manually added):\n",
      "    /usr/local/lib/python3.6/dist-packages/tensorboard/uploader/flags_parser.py\n",
      "Proceed (y/n)? y\n",
      "  Successfully uninstalled tensorboard-2.2.0\n",
      "Collecting tb-nightly\n",
      "  Using cached https://files.pythonhosted.org/packages/a6/63/a0b69e4c814581e52bfa76ec456fe7b910242b9497d8d57732ac1729da53/tb_nightly-2.3.0a20200406-py3-none-any.whl\n",
      "Collecting absl-py>=0.4\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1a/53/9243c600e047bd4c3df9e69cfabc1e8004a82cac2e0c484580a78a94ba2a/absl-py-0.9.0.tar.gz (104kB)\n",
      "\u001b[K     |████████████████████████████████| 112kB 3.4MB/s \n",
      "\u001b[?25hCollecting google-auth-oauthlib<0.5,>=0.4.1\n",
      "  Downloading https://files.pythonhosted.org/packages/7b/b8/88def36e74bee9fce511c9519571f4e485e890093ab7442284f4ffaef60b/google_auth_oauthlib-0.4.1-py2.py3-none-any.whl\n",
      "Collecting protobuf>=3.6.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/57/02/5432412c162989260fab61fa65e0a490c1872739eb91a659896e4d554b26/protobuf-3.11.3-cp36-cp36m-manylinux1_x86_64.whl (1.3MB)\n",
      "\u001b[K     |████████████████████████████████| 1.3MB 9.6MB/s \n",
      "\u001b[?25hCollecting werkzeug>=0.11.15\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/cc/94/5f7079a0e00bd6863ef8f1da638721e9da21e5bacee597595b318f71d62e/Werkzeug-1.0.1-py2.py3-none-any.whl (298kB)\n",
      "\u001b[K     |████████████████████████████████| 307kB 20.3MB/s \n",
      "\u001b[?25hCollecting requests<3,>=2.21.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1a/70/1935c770cb3be6e3a8b78ced23d7e0f3b187f5cbfab4749523ed65d7c9b1/requests-2.23.0-py2.py3-none-any.whl (58kB)\n",
      "\u001b[K     |████████████████████████████████| 61kB 7.2MB/s \n",
      "\u001b[?25hCollecting google-auth<2,>=1.6.3\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/dc/1a/c3c2f3aa4190d8154a146ad33aa5479c8d193cc6211abe5c535921d93389/google_auth-1.13.1-py2.py3-none-any.whl (87kB)\n",
      "\u001b[K     |████████████████████████████████| 92kB 7.4MB/s \n",
      "\u001b[?25hCollecting six>=1.10.0\n",
      "  Downloading https://files.pythonhosted.org/packages/65/eb/1f97cb97bfc2390a276969c6fae16075da282f5058082d4cb10c6c5c1dba/six-1.14.0-py2.py3-none-any.whl\n",
      "Collecting tensorboard-plugin-wit>=1.6.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/51/cd/a0c1f9e4582ea64dddf76c1b808b318d01e3b858a51c715bffab1016ecc7/tensorboard_plugin_wit-1.6.0.post3-py3-none-any.whl (777kB)\n",
      "\u001b[K     |████████████████████████████████| 778kB 24.1MB/s \n",
      "\u001b[?25hCollecting grpcio>=1.24.3\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f3/97/f74da84d4db8cfe95f9b6aa2469be79af1873fec1adb80405105ed99a0a8/grpcio-1.28.1-cp36-cp36m-manylinux2010_x86_64.whl (2.8MB)\n",
      "\u001b[K     |████████████████████████████████| 2.9MB 33.8MB/s \n",
      "\u001b[?25hCollecting wheel>=0.26; python_version >= \"3\"\n",
      "  Downloading https://files.pythonhosted.org/packages/8c/23/848298cccf8e40f5bbb59009b32848a4c38f4e7f3364297ab3c3e2e2cd14/wheel-0.34.2-py2.py3-none-any.whl\n",
      "Collecting markdown>=2.6.8\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ab/c4/ba46d44855e6eb1770a12edace5a165a0c6de13349f592b9036257f3c3d3/Markdown-3.2.1-py2.py3-none-any.whl (88kB)\n",
      "\u001b[K     |████████████████████████████████| 92kB 9.7MB/s \n",
      "\u001b[?25hCollecting setuptools>=41.0.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a0/df/635cdb901ee4a8a42ec68e480c49f85f4c59e8816effbf57d9e6ee8b3588/setuptools-46.1.3-py3-none-any.whl (582kB)\n",
      "\u001b[K     |████████████████████████████████| 583kB 35.9MB/s \n",
      "\u001b[?25hCollecting numpy>=1.12.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/07/08/a549ba8b061005bb629b76adc000f3caaaf881028b963c2e18f811c6edc1/numpy-1.18.2-cp36-cp36m-manylinux1_x86_64.whl (20.2MB)\n",
      "\u001b[K     |████████████████████████████████| 20.2MB 158kB/s \n",
      "\u001b[?25hCollecting requests-oauthlib>=0.7.0\n",
      "  Downloading https://files.pythonhosted.org/packages/a3/12/b92740d845ab62ea4edf04d2f4164d82532b5a0b03836d4d4e71c6f3d379/requests_oauthlib-1.3.0-py2.py3-none-any.whl\n",
      "Collecting chardet<4,>=3.0.2\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8/chardet-3.0.4-py2.py3-none-any.whl (133kB)\n",
      "\u001b[K     |████████████████████████████████| 143kB 36.4MB/s \n",
      "\u001b[?25hCollecting idna<3,>=2.5\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/89/e3/afebe61c546d18fb1709a61bee788254b40e736cff7271c7de5de2dc4128/idna-2.9-py2.py3-none-any.whl (58kB)\n",
      "\u001b[K     |████████████████████████████████| 61kB 7.0MB/s \n",
      "\u001b[?25hCollecting certifi>=2017.4.17\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/57/2b/26e37a4b034800c960a00c4e1b3d9ca5d7014e983e6e729e33ea2f36426c/certifi-2020.4.5.1-py2.py3-none-any.whl (157kB)\n",
      "\u001b[K     |████████████████████████████████| 163kB 31.7MB/s \n",
      "\u001b[?25hCollecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e8/74/6e4f91745020f967d09332bb2b8b9b10090957334692eb88ea4afe91b77f/urllib3-1.25.8-py2.py3-none-any.whl (125kB)\n",
      "\u001b[K     |████████████████████████████████| 133kB 40.7MB/s \n",
      "\u001b[?25hCollecting rsa<4.1,>=3.1.4\n",
      "  Downloading https://files.pythonhosted.org/packages/02/e5/38518af393f7c214357079ce67a317307936896e961e35450b70fad2a9cf/rsa-4.0-py2.py3-none-any.whl\n",
      "Collecting cachetools<5.0,>=2.0.0\n",
      "  Downloading https://files.pythonhosted.org/packages/08/6a/abf83cb951617793fd49c98cb9456860f5df66ff89883c8660aa0672d425/cachetools-4.0.0-py3-none-any.whl\n",
      "Collecting pyasn1-modules>=0.2.1\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/95/de/214830a981892a3e286c3794f41ae67a4495df1108c3da8a9f62159b9a9d/pyasn1_modules-0.2.8-py2.py3-none-any.whl (155kB)\n",
      "\u001b[K     |████████████████████████████████| 163kB 42.3MB/s \n",
      "\u001b[?25hCollecting oauthlib>=3.0.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/05/57/ce2e7a8fa7c0afb54a0581b14a65b56e62b5759dbc98e80627142b8a3704/oauthlib-3.1.0-py2.py3-none-any.whl (147kB)\n",
      "\u001b[K     |████████████████████████████████| 153kB 34.8MB/s \n",
      "\u001b[?25hCollecting pyasn1>=0.1.3\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/62/1e/a94a8d635fa3ce4cfc7f506003548d0a2447ae76fd5ca53932970fe3053f/pyasn1-0.4.8-py2.py3-none-any.whl (77kB)\n",
      "\u001b[K     |████████████████████████████████| 81kB 9.0MB/s \n",
      "\u001b[?25hBuilding wheels for collected packages: absl-py\n",
      "  Building wheel for absl-py (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for absl-py: filename=absl_py-0.9.0-cp36-none-any.whl size=121931 sha256=92b43360db039fe922900e1b88cbda921eb7235b5191d68672057f2d3ad5b2ec\n",
      "  Stored in directory: /root/.cache/pip/wheels/8e/28/49/fad4e7f0b9a1227708cbbee4487ac8558a7334849cb81c813d\n",
      "Successfully built absl-py\n",
      "\u001b[31mERROR: tensorflow 2.2.0rc2 requires tensorboard<2.3.0,>=2.2.0, which is not installed.\u001b[0m\n",
      "\u001b[31mERROR: qtconsole 4.7.2 has requirement pyzmq>=17.1, but you'll have pyzmq 17.0.0 which is incompatible.\u001b[0m\n",
      "\u001b[31mERROR: kaggle 1.5.6 has requirement urllib3<1.25,>=1.21.1, but you'll have urllib3 1.25.8 which is incompatible.\u001b[0m\n",
      "\u001b[31mERROR: google-colab 1.0.0 has requirement google-auth~=1.7.2, but you'll have google-auth 1.13.1 which is incompatible.\u001b[0m\n",
      "\u001b[31mERROR: google-colab 1.0.0 has requirement requests~=2.21.0, but you'll have requests 2.23.0 which is incompatible.\u001b[0m\n",
      "\u001b[31mERROR: google-colab 1.0.0 has requirement six~=1.12.0, but you'll have six 1.14.0 which is incompatible.\u001b[0m\n",
      "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n",
      "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
      "Installing collected packages: six, absl-py, pyasn1, rsa, cachetools, pyasn1-modules, setuptools, google-auth, chardet, idna, certifi, urllib3, requests, oauthlib, requests-oauthlib, google-auth-oauthlib, protobuf, werkzeug, tensorboard-plugin-wit, grpcio, wheel, markdown, numpy, tb-nightly\n",
      "  Found existing installation: six 1.12.0\n",
      "    Uninstalling six-1.12.0:\n",
      "      Successfully uninstalled six-1.12.0\n",
      "  Found existing installation: absl-py 0.9.0\n",
      "    Uninstalling absl-py-0.9.0:\n",
      "      Successfully uninstalled absl-py-0.9.0\n",
      "  Found existing installation: pyasn1 0.4.8\n",
      "    Uninstalling pyasn1-0.4.8:\n",
      "      Successfully uninstalled pyasn1-0.4.8\n",
      "  Found existing installation: rsa 4.0\n",
      "    Uninstalling rsa-4.0:\n",
      "      Successfully uninstalled rsa-4.0\n",
      "  Found existing installation: cachetools 3.1.1\n",
      "    Uninstalling cachetools-3.1.1:\n",
      "      Successfully uninstalled cachetools-3.1.1\n",
      "  Found existing installation: pyasn1-modules 0.2.8\n",
      "    Uninstalling pyasn1-modules-0.2.8:\n",
      "      Successfully uninstalled pyasn1-modules-0.2.8\n",
      "  Found existing installation: setuptools 46.1.3\n",
      "    Uninstalling setuptools-46.1.3:\n",
      "      Successfully uninstalled setuptools-46.1.3\n",
      "  Found existing installation: google-auth 1.7.2\n",
      "    Uninstalling google-auth-1.7.2:\n",
      "      Successfully uninstalled google-auth-1.7.2\n",
      "  Found existing installation: chardet 3.0.4\n",
      "    Uninstalling chardet-3.0.4:\n",
      "      Successfully uninstalled chardet-3.0.4\n",
      "  Found existing installation: idna 2.8\n",
      "    Uninstalling idna-2.8:\n",
      "      Successfully uninstalled idna-2.8\n",
      "  Found existing installation: certifi 2019.11.28\n",
      "    Uninstalling certifi-2019.11.28:\n",
      "      Successfully uninstalled certifi-2019.11.28\n",
      "  Found existing installation: urllib3 1.24.3\n",
      "    Uninstalling urllib3-1.24.3:\n",
      "      Successfully uninstalled urllib3-1.24.3\n",
      "  Found existing installation: requests 2.21.0\n",
      "    Uninstalling requests-2.21.0:\n",
      "      Successfully uninstalled requests-2.21.0\n",
      "  Found existing installation: oauthlib 3.1.0\n",
      "    Uninstalling oauthlib-3.1.0:\n",
      "      Successfully uninstalled oauthlib-3.1.0\n",
      "  Found existing installation: requests-oauthlib 1.3.0\n",
      "    Uninstalling requests-oauthlib-1.3.0:\n",
      "      Successfully uninstalled requests-oauthlib-1.3.0\n",
      "  Found existing installation: google-auth-oauthlib 0.4.1\n",
      "    Uninstalling google-auth-oauthlib-0.4.1:\n",
      "      Successfully uninstalled google-auth-oauthlib-0.4.1\n",
      "  Found existing installation: protobuf 3.10.0\n",
      "    Uninstalling protobuf-3.10.0:\n",
      "      Successfully uninstalled protobuf-3.10.0\n",
      "  Found existing installation: Werkzeug 1.0.1\n",
      "    Uninstalling Werkzeug-1.0.1:\n",
      "      Successfully uninstalled Werkzeug-1.0.1\n",
      "  Found existing installation: tensorboard-plugin-wit 1.6.0.post2\n",
      "    Uninstalling tensorboard-plugin-wit-1.6.0.post2:\n",
      "      Successfully uninstalled tensorboard-plugin-wit-1.6.0.post2\n",
      "  Found existing installation: grpcio 1.27.2\n",
      "    Uninstalling grpcio-1.27.2:\n",
      "      Successfully uninstalled grpcio-1.27.2\n",
      "  Found existing installation: wheel 0.34.2\n",
      "    Uninstalling wheel-0.34.2:\n",
      "      Successfully uninstalled wheel-0.34.2\n",
      "  Found existing installation: Markdown 3.2.1\n",
      "    Uninstalling Markdown-3.2.1:\n",
      "      Successfully uninstalled Markdown-3.2.1\n",
      "  Found existing installation: numpy 1.18.2\n",
      "    Uninstalling numpy-1.18.2:\n",
      "      Successfully uninstalled numpy-1.18.2\n",
      "  Found existing installation: tb-nightly 2.3.0a20200406\n",
      "    Uninstalling tb-nightly-2.3.0a20200406:\n",
      "      Successfully uninstalled tb-nightly-2.3.0a20200406\n",
      "Successfully installed absl-py-0.9.0 cachetools-4.0.0 certifi-2020.4.5.1 chardet-3.0.4 google-auth-1.13.1 google-auth-oauthlib-0.4.1 grpcio-1.28.1 idna-2.9 markdown-3.2.1 numpy-1.18.2 oauthlib-3.1.0 protobuf-3.11.3 pyasn1-0.4.8 pyasn1-modules-0.2.8 requests-2.23.0 requests-oauthlib-1.3.0 rsa-4.0 setuptools-46.1.3 six-1.14.0 tb-nightly-2.3.0a20200406 tensorboard-plugin-wit-1.6.0.post3 urllib3-1.25.8 werkzeug-1.0.1 wheel-0.34.2\n"
     ]
    },
    {
     "data": {
      "application/vnd.colab-display-data+json": {
       "pip_warning": {
        "packages": [
         "cachetools",
         "certifi",
         "chardet",
         "google",
         "grpc",
         "idna",
         "numpy",
         "pkg_resources",
         "pyasn1",
         "pyasn1_modules",
         "requests",
         "rsa",
         "six",
         "urllib3"
        ]
       }
      }
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Thers's some bug of tensorboard in colab so uninstall tensorboard and reinstall tb-nightly.\n",
    "!pip uninstall tensorboard\n",
    "!pip install --force-reinstall tb-nightly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QyvnkoUbuKjx"
   },
   "outputs": [],
   "source": [
    "# 1. Add yourself data processing code in classifier_data_lib.py and create_finetuning_data.py  \n",
    "# 2. Put them into \"models\" directory to replace original files.\n",
    "!cp -ar tf2.0-BERT/code/data/*.py models/official/nlp/data/\n",
    "!cp -ar tf2.0-BERT/code/bert/*.py models/official/nlp/bert/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 119
    },
    "colab_type": "code",
    "id": "pQ3jtVTEkZ2S",
    "outputId": "24282624-7bc6-418b-f8f3-17b5fd0b7ad9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: PYTHONPATH=./models\n",
      "env: TASK_NAME=WEIBO\n",
      "env: CONFIG_FILE=tf2.0-BERT/process_dir/models/hub_config\n",
      "env: DATA_DIR=tf2.0-BERT/process_dir/dataset/train_eval_test_data\n",
      "env: OUTPUT_DIR=tf2.0-BERT/output\n",
      "env: MODEL_HUB_URL=https://tfhub.dev/tensorflow/bert_zh_L-12_H-768_A-12/1\n"
     ]
    }
   ],
   "source": [
    "# Set environment variables.\n",
    "%set_env PYTHONPATH=./models\n",
    "%set_env TASK_NAME=WEIBO\n",
    "%set_env CONFIG_FILE=tf2.0-BERT/process_dir/models/hub_config\n",
    "%set_env DATA_DIR=tf2.0-BERT/process_dir/dataset/train_eval_test_data\n",
    "%set_env OUTPUT_DIR=tf2.0-BERT/output\n",
    "%set_env MODEL_HUB_URL=https://tfhub.dev/tensorflow/bert_zh_L-12_H-768_A-12/1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "yyDamUNDfCg0",
    "outputId": "7911af16-b0d7-4b26-a5a4-bb8778af2b09"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-08 09:36:23.137346: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1\n",
      "I0408 09:36:26.799381 139964357027712 classifier_data_lib.py:534] Writing example 0 of 89827\n",
      "I0408 09:36:26.800556 139964357027712 classifier_data_lib.py:507] *** Example ***\n",
      "I0408 09:36:26.800692 139964357027712 classifier_data_lib.py:508] guid: train-1\n",
      "I0408 09:36:26.801644 139964357027712 classifier_data_lib.py:510] tokens: [CLS] # 上 海 肺 炎 # 近 期 上 海 也 出 现 了 一 例 新 型 冠 状 病 毒 感 染 的 肺 炎 疑 似 患 者 ， 小 伙 伴 们 出 门 要 记 得 带 好 口 罩 ， 少 去 人 员 密 集 的 场 所 噢 ！ 注 意 居 室 和 办 公 室 通 风 ， 多 多 锻 炼 身 体 ， 增 强 抵 抗 力 ， 健 健 康 康 的 哈 ～ ? 牙 齿 矫 正 李 彦 昂 医 生 发 布 于 疫 情 恐 慌 有 图 有 视 频 [SEP]\n",
      "I0408 09:36:26.801824 139964357027712 classifier_data_lib.py:511] input_ids: 101 108 677 3862 5511 4142 108 6818 3309 677 3862 738 1139 4385 749 671 891 3173 1798 1094 4307 4567 3681 2697 3381 4638 5511 4142 4542 849 2642 5442 8024 2207 832 845 812 1139 7305 6206 6381 2533 2372 1962 1366 5388 8024 2208 1343 782 1447 2166 7415 4638 1767 2792 1688 8013 3800 2692 2233 2147 1469 1215 1062 2147 6858 7599 8024 1914 1914 7248 4159 6716 860 8024 1872 2487 2850 2834 1213 8024 978 978 2434 2434 4638 1506 8080 136 4280 7976 4763 3633 3330 2504 3203 1278 4495 1355 2357 754 4554 2658 2607 2707 3300 1745 3300 6228 7574 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:36:26.802001 139964357027712 classifier_data_lib.py:512] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:36:26.802175 139964357027712 classifier_data_lib.py:513] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:36:26.802295 139964357027712 classifier_data_lib.py:514] label: 0 (id = 1)\n",
      "I0408 09:36:26.803714 139964357027712 classifier_data_lib.py:507] *** Example ***\n",
      "I0408 09:36:26.803832 139964357027712 classifier_data_lib.py:508] guid: train-2\n",
      "I0408 09:36:26.804028 139964357027712 classifier_data_lib.py:510] tokens: [CLS] / / @ : 无 症 状 感 染 注 意 ， 如 果 有 密 切 接 触 者 确 诊 ， 自 己 尚 无 症 状 的 也 不 能 掉 以 轻 心 。 另 外 [UNK] 既 然 您 只 相 信 医 学 文 献 报 道 ， 那 就 没 办 法 了 [UNK] 震 撼 我 心 查 看 图 片 / / @ : 新 型 冠 状 病 毒 ， 无 症 状 感 染 是 存 在 的 。 sars 病 例 也 有 类 似 情 况 。 这 些 无 症 状 的 感 染 者 ， 还 有 可 能 传 染 别 人 。 _ li ##ch ##ou ##ni 发 布 于 疫 情 高 峰 无 图 无 视 频 [SEP]\n",
      "I0408 09:36:26.804214 139964357027712 classifier_data_lib.py:511] input_ids: 101 120 120 137 131 3187 4568 4307 2697 3381 3800 2692 8024 1963 3362 3300 2166 1147 2970 6239 5442 4802 6402 8024 5632 2346 2213 3187 4568 4307 4638 738 679 5543 2957 809 6768 2552 511 1369 1912 100 3188 4197 2644 1372 4685 928 1278 2110 3152 4346 2845 6887 8024 6929 2218 3766 1215 3791 749 100 7448 3072 2769 2552 3389 4692 1745 4275 120 120 137 131 3173 1798 1094 4307 4567 3681 8024 3187 4568 4307 2697 3381 3221 2100 1762 4638 511 12390 4567 891 738 3300 5102 849 2658 1105 511 6821 763 3187 4568 4307 4638 2697 3381 5442 8024 6820 3300 1377 5543 837 3381 1166 782 511 142 9341 8370 9857 8833 1355 2357 754 4554 2658 7770 2292 3187 1745 3187 6228 7574 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:36:26.804400 139964357027712 classifier_data_lib.py:512] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:36:26.804581 139964357027712 classifier_data_lib.py:513] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:36:26.804711 139964357027712 classifier_data_lib.py:514] label: 0 (id = 1)\n",
      "I0408 09:36:26.805539 139964357027712 classifier_data_lib.py:507] *** Example ***\n",
      "I0408 09:36:26.805676 139964357027712 classifier_data_lib.py:508] guid: train-3\n",
      "I0408 09:36:26.805816 139964357027712 classifier_data_lib.py:510] tokens: [CLS] # 山 东 副 省 长 带 队 援 鄂 # 不 止 副 省 长 ， 省 委 副 书 记 也 去 了 ！ 山 东 太 硬 核 了 ！ ? 幸 福 的 紫 色 豌 豆 花 发 布 于 平 复 期 有 图 有 视 频 [SEP]\n",
      "I0408 09:36:26.805988 139964357027712 classifier_data_lib.py:511] input_ids: 101 108 2255 691 1199 4689 7270 2372 7339 3001 6964 108 679 3632 1199 4689 7270 8024 4689 1999 1199 741 6381 738 1343 749 8013 2255 691 1922 4801 3417 749 8013 136 2401 4886 4638 5166 5682 6491 6486 5709 1355 2357 754 2398 1908 3309 3300 1745 3300 6228 7574 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:36:26.806164 139964357027712 classifier_data_lib.py:512] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:36:26.806335 139964357027712 classifier_data_lib.py:513] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:36:26.806439 139964357027712 classifier_data_lib.py:514] label: 1 (id = 2)\n",
      "I0408 09:36:26.807158 139964357027712 classifier_data_lib.py:507] *** Example ***\n",
      "I0408 09:36:26.807271 139964357027712 classifier_data_lib.py:508] guid: train-4\n",
      "I0408 09:36:26.807399 139964357027712 classifier_data_lib.py:510] tokens: [CLS] 吃 蝙 蝠 的 给 老 子 死 ！ ！ ！ ！ ！ ！ 牙 疼 死 了 不 敢 去 医 院 ? 新 垣 橘 发 布 于 平 复 期 无 图 无 视 频 [SEP]\n",
      "I0408 09:36:26.807566 139964357027712 classifier_data_lib.py:511] input_ids: 101 1391 6073 6075 4638 5314 5439 2094 3647 8013 8013 8013 8013 8013 8013 4280 4563 3647 749 679 3140 1343 1278 7368 136 3173 1804 3580 1355 2357 754 2398 1908 3309 3187 1745 3187 6228 7574 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:36:26.807753 139964357027712 classifier_data_lib.py:512] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:36:26.807934 139964357027712 classifier_data_lib.py:513] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:36:26.808039 139964357027712 classifier_data_lib.py:514] label: -1 (id = 0)\n",
      "I0408 09:36:26.808654 139964357027712 classifier_data_lib.py:507] *** Example ***\n",
      "I0408 09:36:26.808768 139964357027712 classifier_data_lib.py:508] guid: train-5\n",
      "I0408 09:36:26.808897 139964357027712 classifier_data_lib.py:510] tokens: [CLS] [ 爱 你 ] / / @ : # 陪 黄 景 瑜 一 起 慢 慢 走 # 最 美 逆 行 者 感 恩 白 衣 天 使 @ [SEP]\n",
      "I0408 09:36:26.809067 139964357027712 classifier_data_lib.py:511] input_ids: 101 138 4263 872 140 120 120 137 131 108 7373 7942 3250 4447 671 6629 2714 2714 6624 108 3297 5401 6847 6121 5442 2697 2617 4635 6132 1921 886 137 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:36:26.809245 139964357027712 classifier_data_lib.py:512] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:36:26.809461 139964357027712 classifier_data_lib.py:513] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:36:26.809564 139964357027712 classifier_data_lib.py:514] label: 0 (id = 1)\n",
      "I0408 09:36:35.789376 139964357027712 classifier_data_lib.py:534] Writing example 10000 of 89827\n",
      "I0408 09:36:44.733593 139964357027712 classifier_data_lib.py:534] Writing example 20000 of 89827\n",
      "I0408 09:36:53.746362 139964357027712 classifier_data_lib.py:534] Writing example 30000 of 89827\n",
      "I0408 09:37:02.899731 139964357027712 classifier_data_lib.py:534] Writing example 40000 of 89827\n",
      "I0408 09:37:11.906952 139964357027712 classifier_data_lib.py:534] Writing example 50000 of 89827\n",
      "I0408 09:37:20.701751 139964357027712 classifier_data_lib.py:534] Writing example 60000 of 89827\n",
      "I0408 09:37:29.851054 139964357027712 classifier_data_lib.py:534] Writing example 70000 of 89827\n",
      "I0408 09:37:38.980506 139964357027712 classifier_data_lib.py:534] Writing example 80000 of 89827\n",
      "I0408 09:37:48.004768 139964357027712 classifier_data_lib.py:534] Writing example 0 of 10000\n",
      "I0408 09:37:48.005590 139964357027712 classifier_data_lib.py:507] *** Example ***\n",
      "I0408 09:37:48.005755 139964357027712 classifier_data_lib.py:508] guid: dev-1\n",
      "I0408 09:37:48.005923 139964357027712 classifier_data_lib.py:510] tokens: [CLS] 不 堪 阴 谋 论 热 传 ， 美 国 宣 布 彻 查 新 冠 病 毒 起 源 ！ 石 正 丽 也 急 盼 清 白 o 不 堪 阴 谋 论 热 传 ， 美 国 宣 布 彻 查 新 冠 病 毒 起 源 ！ 石 正 丽 也 急 盼 清 白 ? 雨 后 - 天 青 发 布 于 平 复 期 有 图 有 视 频 [SEP]\n",
      "I0408 09:37:48.006119 139964357027712 classifier_data_lib.py:511] input_ids: 101 679 1838 7346 6450 6389 4178 837 8024 5401 1744 2146 2357 2515 3389 3173 1094 4567 3681 6629 3975 8013 4767 3633 714 738 2593 4687 3926 4635 157 679 1838 7346 6450 6389 4178 837 8024 5401 1744 2146 2357 2515 3389 3173 1094 4567 3681 6629 3975 8013 4767 3633 714 738 2593 4687 3926 4635 136 7433 1400 118 1921 7471 1355 2357 754 2398 1908 3309 3300 1745 3300 6228 7574 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:37:48.006302 139964357027712 classifier_data_lib.py:512] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:37:48.006482 139964357027712 classifier_data_lib.py:513] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:37:48.006592 139964357027712 classifier_data_lib.py:514] label: 0 (id = 1)\n",
      "I0408 09:37:48.007652 139964357027712 classifier_data_lib.py:507] *** Example ***\n",
      "I0408 09:37:48.007771 139964357027712 classifier_data_lib.py:508] guid: dev-2\n",
      "I0408 09:37:48.007924 139964357027712 classifier_data_lib.py:510] tokens: [CLS] / / @ : 痘 痘 一 般 不 建 议 挤 。 如 果 非 要 挤 ， 那 么 未 引 起 炎 症 的 黑 头 、 闭 口 ， 以 及 面 部 危 险 三 角 区 以 外 的 脓 疱 型 痘 痘 （ 露 白 头 ） 可 以 挤 ， 但 一 定 要 做 好 消 毒 ~ 我 代 k 先 生 回 信 发 布 于 疫 情 早 期 无 图 无 视 频 [SEP]\n",
      "I0408 09:37:48.008105 139964357027712 classifier_data_lib.py:511] input_ids: 101 120 120 137 131 4576 4576 671 5663 679 2456 6379 2915 511 1963 3362 7478 6206 2915 8024 6929 720 3313 2471 6629 4142 4568 4638 7946 1928 510 7308 1366 8024 809 1350 7481 6956 1314 7372 676 6235 1277 809 1912 4638 5555 4557 1798 4576 4576 8020 7463 4635 1928 8021 1377 809 2915 8024 852 671 2137 6206 976 1962 3867 3681 172 2769 807 153 1044 4495 1726 928 1355 2357 754 4554 2658 3193 3309 3187 1745 3187 6228 7574 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:37:48.008280 139964357027712 classifier_data_lib.py:512] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:37:48.008455 139964357027712 classifier_data_lib.py:513] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:37:48.008560 139964357027712 classifier_data_lib.py:514] label: 0 (id = 1)\n",
      "I0408 09:37:48.009403 139964357027712 classifier_data_lib.py:507] *** Example ***\n",
      "I0408 09:37:48.009519 139964357027712 classifier_data_lib.py:508] guid: dev-3\n",
      "I0408 09:37:48.009688 139964357027712 classifier_data_lib.py:510] tokens: [CLS] 肥 宅 生 活 打 卡 ～ ##dn 郑 多 燕 d ##n 林 芊 妤 d ##n 周 六 野 d ##n 练 腿 d ##n 低 碳 anna ##mc ##n ##ult ##y 地 狱 拉 伸 第 n 天 ? ? ? ? ? ir ##is _ xx ##n 发 布 于 平 复 期 有 图 有 视 频 [SEP]\n",
      "I0408 09:37:48.009863 139964357027712 classifier_data_lib.py:511] input_ids: 101 5503 2125 4495 3833 2802 1305 8080 10853 6948 1914 4242 146 8171 3360 5691 1979 146 8171 1453 1063 7029 146 8171 5298 5597 146 8171 856 4823 9779 10307 8171 9766 8179 1765 4328 2861 847 5018 156 1921 136 136 136 136 136 12410 8331 142 8584 8171 1355 2357 754 2398 1908 3309 3300 1745 3300 6228 7574 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:37:48.010037 139964357027712 classifier_data_lib.py:512] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:37:48.010209 139964357027712 classifier_data_lib.py:513] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:37:48.010311 139964357027712 classifier_data_lib.py:514] label: 0 (id = 1)\n",
      "I0408 09:37:48.011244 139964357027712 classifier_data_lib.py:507] *** Example ***\n",
      "I0408 09:37:48.011361 139964357027712 classifier_data_lib.py:508] guid: dev-4\n",
      "I0408 09:37:48.011513 139964357027712 classifier_data_lib.py:510] tokens: [CLS] 近 段 时 间 ， 疫 情 牵 动 着 全 国 人 民 的 心 ， 不 少 明 星 比 如 韩 红 、 黄 晓 明 、 赵 本 山 等 明 星 、 网 红 ， 纷 纷 自 发 捐 款 捐 物 ， 其 中 韩 红 的 表 现 更 令 人 佩 服 不 已 ！ ? 跟 着 我 到 远 方 发 布 于 平 复 期 有 图 有 视 频 [SEP]\n",
      "I0408 09:37:48.011709 139964357027712 classifier_data_lib.py:511] input_ids: 101 6818 3667 3198 7313 8024 4554 2658 4293 1220 4708 1059 1744 782 3696 4638 2552 8024 679 2208 3209 3215 3683 1963 7506 5273 510 7942 3236 3209 510 6627 3315 2255 5023 3209 3215 510 5381 5273 8024 5290 5290 5632 1355 2935 3621 2935 4289 8024 1071 704 7506 5273 4638 6134 4385 3291 808 782 877 3302 679 2347 8013 136 6656 4708 2769 1168 6823 3175 1355 2357 754 2398 1908 3309 3300 1745 3300 6228 7574 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:37:48.011882 139964357027712 classifier_data_lib.py:512] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:37:48.012056 139964357027712 classifier_data_lib.py:513] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:37:48.012159 139964357027712 classifier_data_lib.py:514] label: 1 (id = 2)\n",
      "I0408 09:37:48.013424 139964357027712 classifier_data_lib.py:507] *** Example ***\n",
      "I0408 09:37:48.013540 139964357027712 classifier_data_lib.py:508] guid: dev-5\n",
      "I0408 09:37:48.013710 139964357027712 classifier_data_lib.py:510] tokens: [CLS] 【 嘴 巴 告 诉 你 】 ① 嘴 角 裂 ： 脾 有 火 肺 气 不 足 ， 绿 豆 百 合 汤 ； ② 嘴 唇 干 ： 肝 肾 阴 虚 ， 枸 杞 、 麦 冬 泡 水 喝 ； ③ 口 气 大 ： 龋 齿 、 牙 周 或 胃 肠 湿 热 ， 菊 花 或 金 银 花 泡 茶 ； ④ 流 口 水 ： 脾 肾 虚 ， 少 吃 肉 多 吃 枣 。 ⑤ 唇 色 浅 ： 气 虚 ， 含 参 片 吃 红 枣 ； ⑥ 唇 色 紫 ： 血 瘀 、 氧 气 不 足 ， 吃 姜 、 常 深 呼 吸 、 不 熬 夜 ? 老 蔡 说 健 康 发 布 于 疫 情 高 峰 无 图 无 视 频 [SEP]\n",
      "I0408 09:37:48.013877 139964357027712 classifier_data_lib.py:511] input_ids: 101 523 1673 2349 1440 6401 872 524 405 1673 6235 6162 8038 5569 3300 4125 5511 3698 679 6639 8024 5344 6486 4636 1394 3739 8039 406 1673 1535 2397 8038 5498 5513 7346 5994 8024 3375 3337 510 7931 1100 3796 3717 1600 8039 407 1366 3698 1920 8038 7981 7976 510 4280 1453 2772 5517 5499 3969 4178 8024 5825 5709 2772 7032 7213 5709 3796 5763 8039 408 3837 1366 3717 8038 5569 5513 5994 8024 2208 1391 5489 1914 1391 3365 511 409 1535 5682 3840 8038 3698 5994 8024 1419 1346 4275 1391 5273 3365 8039 410 1535 5682 5166 8038 6117 4595 510 3709 3698 679 6639 8024 1391 2002 510 2382 3918 1461 1429 510 679 4228 1915 136 5439 5918 6432 978 2434 1355 2357 754 4554 2658 7770 2292 3187 1745 3187 6228 7574 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:37:48.014050 139964357027712 classifier_data_lib.py:512] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:37:48.014221 139964357027712 classifier_data_lib.py:513] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "I0408 09:37:48.014325 139964357027712 classifier_data_lib.py:514] label: 0 (id = 1)\n"
     ]
    }
   ],
   "source": [
    "# Create train and eval data with tf_record format.\n",
    "!python ./models/official/nlp/data/create_finetuning_data.py \\\n",
    "    --input_data_dir=${DATA_DIR} \\\n",
    "    --vocab_file=${CONFIG_FILE}/vocab.txt \\\n",
    "    --train_data_output_path=${DATA_DIR}/${TASK_NAME}_train.tf_record \\\n",
    "    --eval_data_output_path=${DATA_DIR}/${TASK_NAME}_eval.tf_record \\\n",
    "    --meta_data_file_path=${DATA_DIR}/${TASK_NAME}_meta_data \\\n",
    "    --fine_tuning_task_type=classification \\\n",
    "    --max_seq_length=180 \\\n",
    "    --classification_task_name=${TASK_NAME}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "w3iSdrvClTCD",
    "outputId": "720bdf57-11eb-4e88-94c7-3d48ea0bf2a9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-08 09:43:28.746828: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1\n",
      "/usr/local/lib/python3.6/dist-packages/absl/flags/_validators.py:359: UserWarning: Flag --model_dir has a non-None default value; therefore, mark_flag_as_required will pass even if flag is not specified in the command line!\n",
      "  'command line!' % flag_name)\n",
      "2020-04-08 09:43:31.096704: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1\n",
      "2020-04-08 09:43:31.121869: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-04-08 09:43:31.122693: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1544] Found device 0 with properties: \n",
      "pciBusID: 0000:00:04.0 name: Tesla K80 computeCapability: 3.7\n",
      "coreClock: 0.8235GHz coreCount: 13 deviceMemorySize: 11.17GiB deviceMemoryBandwidth: 223.96GiB/s\n",
      "2020-04-08 09:43:31.122749: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1\n",
      "2020-04-08 09:43:31.122861: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10\n",
      "2020-04-08 09:43:31.122922: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10\n",
      "2020-04-08 09:43:31.123008: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10\n",
      "2020-04-08 09:43:31.123073: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10\n",
      "2020-04-08 09:43:31.124169: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10\n",
      "2020-04-08 09:43:31.124264: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7\n",
      "2020-04-08 09:43:31.124412: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-04-08 09:43:31.125289: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-04-08 09:43:31.125969: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1686] Adding visible gpu devices: 0\n",
      "2020-04-08 09:43:31.126384: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with Intel(R) MKL-DNN to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2020-04-08 09:43:31.132107: I tensorflow/core/platform/profile_utils/cpu_utils.cc:102] CPU Frequency: 2300000000 Hz\n",
      "2020-04-08 09:43:31.132378: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x2fc1b80 initialized for platform Host (this does not guarantee that XLA will be used). Devices:\n",
      "2020-04-08 09:43:31.132421: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version\n",
      "2020-04-08 09:43:31.192028: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-04-08 09:43:31.193334: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x2fc0f40 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "2020-04-08 09:43:31.193391: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Tesla K80, Compute Capability 3.7\n",
      "2020-04-08 09:43:31.193941: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-04-08 09:43:31.194837: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1544] Found device 0 with properties: \n",
      "pciBusID: 0000:00:04.0 name: Tesla K80 computeCapability: 3.7\n",
      "coreClock: 0.8235GHz coreCount: 13 deviceMemorySize: 11.17GiB deviceMemoryBandwidth: 223.96GiB/s\n",
      "2020-04-08 09:43:31.194947: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1\n",
      "2020-04-08 09:43:31.195003: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10\n",
      "2020-04-08 09:43:31.195050: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10\n",
      "2020-04-08 09:43:31.195106: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10\n",
      "2020-04-08 09:43:31.195148: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10\n",
      "2020-04-08 09:43:31.195232: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10\n",
      "2020-04-08 09:43:31.195273: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7\n",
      "2020-04-08 09:43:31.195478: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-04-08 09:43:31.196469: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-04-08 09:43:31.197259: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1686] Adding visible gpu devices: 0\n",
      "2020-04-08 09:43:31.197357: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1\n",
      "2020-04-08 09:43:31.725984: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1085] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2020-04-08 09:43:31.726054: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1091]      0 \n",
      "2020-04-08 09:43:31.726074: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1104] 0:   N \n",
      "2020-04-08 09:43:31.726381: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-04-08 09:43:31.727284: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-04-08 09:43:31.728030: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:39] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n",
      "2020-04-08 09:43:31.728094: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1230] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10621 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)\n",
      "INFO:tensorflow:Using MirroredStrategy with devices ('/job:localhost/replica:0/task:0/device:GPU:0',)\n",
      "I0408 09:43:31.730195 140491766704000 mirrored_strategy.py:341] Using MirroredStrategy with devices ('/job:localhost/replica:0/task:0/device:GPU:0',)\n",
      "I0408 09:43:31.732111 140491766704000 run_classifier.py:173] Training using TF 2.0 Keras compile/fit API with distribution strategy.\n",
      "I0408 09:43:32.064090 140491766704000 resolver.py:79] Using /tmp/tfhub_modules to cache modules.\n",
      "I0408 09:43:43.635419 140491766704000 optimization.py:87] using Adamw optimizer\n",
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_word_ids (InputLayer)     [(None, 180)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_mask (InputLayer)         [(None, 180)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_type_ids (InputLayer)     [(None, 180)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "keras_layer (KerasLayer)        [(None, 768), (None, 102267649   input_word_ids[0][0]             \n",
      "                                                                 input_mask[0][0]                 \n",
      "                                                                 input_type_ids[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 768)          0           keras_layer[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "output (Dense)                  (None, 3)            2307        dropout[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 102,269,956\n",
      "Trainable params: 102,269,955\n",
      "Non-trainable params: 1\n",
      "__________________________________________________________________________________________________\n",
      "2020-04-08 09:43:43.671707: I tensorflow/core/profiler/lib/profiler_session.cc:145] Profiler session started.\n",
      "2020-04-08 09:43:43.671798: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1372] Profiler found 1 GPUs\n",
      "2020-04-08 09:43:43.672785: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcupti.so.10.1\n",
      "2020-04-08 09:43:43.738323: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1490] CUPTI activity buffer flushed\n",
      "2020-04-08 09:43:44.284082: W tensorflow/core/kernels/data/captured_function.cc:458] Disabling multi-device execution for a function that uses the experimental_ints_on_device attribute.\n",
      "2020-04-08 09:43:44.285358: W tensorflow/core/kernels/data/captured_function.cc:458] Disabling multi-device execution for a function that uses the experimental_ints_on_device attribute.\n",
      "Epoch 1/2\n",
      "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
      "I0408 09:43:56.253382 140491766704000 cross_device_ops.py:440] Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
      "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
      "I0408 09:43:56.255774 140491766704000 cross_device_ops.py:440] Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
      "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
      "I0408 09:43:56.260389 140491766704000 cross_device_ops.py:440] Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
      "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
      "I0408 09:43:56.262510 140491766704000 cross_device_ops.py:440] Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
      "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
      "I0408 09:43:59.064546 140491766704000 cross_device_ops.py:440] Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
      "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
      "I0408 09:43:59.068442 140491766704000 cross_device_ops.py:440] Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
      "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
      "I0408 09:43:59.073513 140491766704000 cross_device_ops.py:440] Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
      "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
      "I0408 09:43:59.075803 140491766704000 cross_device_ops.py:440] Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
      "2020-04-08 09:44:05.414428: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10\n",
      "   1/2807 [..............................] - ETA: 0s - accuracy: 0.5625 - loss: 1.00712020-04-08 09:44:08.291771: I tensorflow/core/profiler/lib/profiler_session.cc:145] Profiler session started.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/summary_ops_v2.py:1271: stop (from tensorflow.python.eager.profiler) is deprecated and will be removed after 2020-07-01.\n",
      "Instructions for updating:\n",
      "use `tf.profiler.experimental.stop` instead.\n",
      "W0408 09:44:10.902147 140491766704000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/summary_ops_v2.py:1271: stop (from tensorflow.python.eager.profiler) is deprecated and will be removed after 2020-07-01.\n",
      "Instructions for updating:\n",
      "use `tf.profiler.experimental.stop` instead.\n",
      "2020-04-08 09:44:10.907596: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1490] CUPTI activity buffer flushed\n",
      "2020-04-08 09:44:10.972003: I tensorflow/core/profiler/internal/gpu/device_tracer.cc:217]  GpuTracer has collected 2555 callback api events and 2555 activity events.\n",
      "2204/2807 [======================>.......] - ETA: 26:35 - accuracy: 0.7291 - loss: 0.6139"
     ]
    }
   ],
   "source": [
    "# Start to train and eval...\n",
    "!python ./models/official/nlp/bert/run_classifier.py \\\n",
    "    --mode='train_and_eval' \\\n",
    "    --input_meta_data_path=${DATA_DIR}/${TASK_NAME}_meta_data \\\n",
    "    --train_data_path=${DATA_DIR}/${TASK_NAME}_train.tf_record \\\n",
    "    --eval_data_path=${DATA_DIR}/${TASK_NAME}_eval.tf_record \\\n",
    "    --bert_config_file=${CONFIG_FILE}/bert_config.json \\\n",
    "    --train_batch_size=32 \\\n",
    "    --eval_batch_size=32 \\\n",
    "    --steps_per_loop=1 \\\n",
    "    --learning_rate=3e-5 \\\n",
    "    --num_train_epochs=2 \\\n",
    "    --model_dir=${OUTPUT_DIR} \\\n",
    "    --distribution_strategy=mirrored \\\n",
    "    --hub_module_url=${MODEL_HUB_URL} \\\n",
    "    --use_keras_compile_fit=True \\\n",
    "    --num_gpus=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yp01cVO6XzWd"
   },
   "outputs": [],
   "source": [
    "# generate predict dataset\n",
    "!python ./models/official/nlp/data/do_pred_data.py \\\n",
    "    --input_data_dir=tf2.0-BERT/process_dir/dataset/predict_data \\\n",
    "    --predict_data_output_path=tf2.0-BERT/process_dir/dataset/predict_data/${TASK_NAME}_pred.tf_record \\\n",
    "    --vocab_file=${CONFIG_FILE}/vocab.txt \\\n",
    "    --meta_data_file_path=tf2.0-BERT/process_dir/dataset/predict_data/${TASK_NAME}_meta_data \\\n",
    "    --fine_tuning_task_type=classification \\\n",
    "    --max_seq_length=180 \\\n",
    "    --classification_task_name=${TASK_NAME}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rzQOzzXICkHM"
   },
   "outputs": [],
   "source": [
    "# do predict!\n",
    "!python ./models/official/nlp/bert/do_predict.py \\\n",
    "  --predict_data_path=tf2.0-BERT/process_dir/dataset/predict_data/${TASK_NAME}_pred.tf_record \\\n",
    "  --predict_output_dir=tf2.0-BERT/process_dir/dataset/predict_data/predict \\\n",
    "  --input_meta_data_path=tf2.0-BERT/process_dir/dataset/predict_data/${TASK_NAME}_meta_data \\\n",
    "  --predict_batch_size=32 \\\n",
    "  --max_seq_length=180 \\\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qHLAQLHe7uop"
   },
   "source": [
    "# 新段落"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 462
    },
    "colab_type": "code",
    "id": "2Io8ZnMQ5BB2",
    "outputId": "dfb79953-45e8-49e3-e9dd-30b7932741c1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ERROR: Failed to launch TensorBoard (exited with 1).\n",
       "Contents of stderr:\n",
       "2020-04-08 04:33:22.994026: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1\n",
       "Traceback (most recent call last):\n",
       "  File \"/usr/local/bin/tensorboard\", line 8, in <module>\n",
       "    sys.exit(run_main())\n",
       "  File \"/usr/local/lib/python3.6/dist-packages/tensorboard/main.py\", line 75, in run_main\n",
       "    app.run(tensorboard.main, flags_parser=tensorboard.configure)\n",
       "  File \"/usr/local/lib/python3.6/dist-packages/absl/app.py\", line 299, in run\n",
       "    _run_main(main, args)\n",
       "  File \"/usr/local/lib/python3.6/dist-packages/absl/app.py\", line 250, in _run_main\n",
       "    sys.exit(main(argv))\n",
       "  File \"/usr/local/lib/python3.6/dist-packages/tensorboard/program.py\", line 289, in main\n",
       "    return runner(self.flags) or 0\n",
       "  File \"/usr/local/lib/python3.6/dist-packages/tensorboard/program.py\", line 305, in _run_serve_subcommand\n",
       "    server = self._make_server()\n",
       "  File \"/usr/local/lib/python3.6/dist-packages/tensorboard/program.py\", line 409, in _make_server\n",
       "    self.flags, self.plugin_loaders, self.assets_zip_provider\n",
       "  File \"/usr/local/lib/python3.6/dist-packages/tensorboard/backend/application.py\", line 183, in standard_tensorboard_wsgi\n",
       "    flags, plugin_loaders, data_provider, assets_zip_provider, multiplexer\n",
       "  File \"/usr/local/lib/python3.6/dist-packages/tensorboard/backend/application.py\", line 264, in TensorBoardWSGIApp\n",
       "    tbplugins, flags.path_prefix, data_provider, experimental_plugins\n",
       "  File \"/usr/local/lib/python3.6/dist-packages/tensorboard/backend/application.py\", line 337, in __init__\n",
       "    \"Duplicate plugins for name %s\" % plugin.plugin_name\n",
       "ValueError: Duplicate plugins for name projector"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# view summaries in tensorboard\n",
    "%reload_ext tensorboard\n",
    "%tensorboard --logdir=${OUTPUT_DIR}/summaries/ "
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "bert-tf2-zh-demo.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
